package com.dxf.bigdata.D02_demo1

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object S03_WordCount {

  def doSome(sc: SparkContext) = {
    //1 读取文件,获取一行一行的数据
    val lines = sc.textFile("datas/*")
    //2 空格切分
    val words = lines.flatMap(_.split(" "))

    val wordToOne: RDD[(String, Int)] = words.map(
      word => (word, 1)
    )

    //spark 分组+聚合提供了更加方法的方法
    //    wordToOne.reduceByKey((a,b)=>{a+b})
    //    wordToOne.reduceByKey((a,b)=>a+b)
    var wordToCount = wordToOne.reduceByKey(_ + _) //    等于  wordToOne.reduceByKey(_+_)


    val array: Array[(String, Int)] = wordToCount.collect()

    array.foreach(println)
  }

  def main(args: Array[String]): Unit = {

    //1 建立和spark的连接

    val sparkConf = new SparkConf().setMaster("local").setAppName("wordCount")
    val sc = new SparkContext(sparkConf)


    //2 处理业务
    doSome(sc)

    //3 关闭连接
    sc.stop()


  }
}
